{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入必要的工具包\n",
    "# 数据读取及基本处理\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "import matplotlib as pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.866045</td>\n",
       "      <td>-0.031990</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>0.166619</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.205066</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-0.852200</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>2.016662</td>\n",
       "      <td>-0.693761</td>\n",
       "      <td>-0.012301</td>\n",
       "      <td>-0.181541</td>\n",
       "      <td>-1.332500</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.073567</td>\n",
       "      <td>-0.528319</td>\n",
       "      <td>-0.695245</td>\n",
       "      <td>-0.540642</td>\n",
       "      <td>-0.633881</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504422</td>\n",
       "      <td>-2.679076</td>\n",
       "      <td>0.670643</td>\n",
       "      <td>0.316566</td>\n",
       "      <td>1.549303</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies   Glucose  BloodPressure  SkinThickness   Insulin       BMI  \\\n",
       "0     0.639947  0.866045      -0.031990       0.670643 -0.181541  0.166619   \n",
       "1    -0.844885 -1.205066      -0.528319      -0.012301 -0.181541 -0.852200   \n",
       "2     1.233880  2.016662      -0.693761      -0.012301 -0.181541 -1.332500   \n",
       "3    -0.844885 -1.073567      -0.528319      -0.695245 -0.540642 -0.633881   \n",
       "4    -1.141852  0.504422      -2.679076       0.670643  0.316566  1.549303   \n",
       "\n",
       "   DiabetesPedigreeFunction       Age  Outcome  \n",
       "0                  0.468492  1.425995        1  \n",
       "1                 -0.365061 -0.190672        0  \n",
       "2                  0.604397 -0.105584        1  \n",
       "3                 -0.920763 -1.041549        0  \n",
       "4                  5.484909 -0.020496        1  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 读入数据\n",
    "train = pd.read_csv(\"FE_pima-indians-diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 768 non-null float64\n",
      "Glucose                     768 non-null float64\n",
      "BloodPressure               768 non-null float64\n",
      "SkinThickness               768 non-null float64\n",
      "Insulin                     768 non-null float64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null float64\n",
      "Outcome                     768 non-null int64\n",
      "dtypes: float64(8), int64(1)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#  get labels\n",
    "y_train = train['Outcome']   \n",
    "X_train = train.drop([\"Outcome\"], axis=1)\n",
    "# 数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "X_train = ss_X.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "# 1. 生成学习器实例\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is: [-0.48797856 -0.53011593 -0.4562292  -0.422546   -0.48392885]\n",
      "cv logloss is : -0.47615970944434044\n"
     ]
    }
   ],
   "source": [
    "# 交叉验证用于评估模型性能和进行参数调优\n",
    "from sklearn.model_selection import cross_val_score\n",
    "loss =  cross_val_score(lr, X_train, y_train, cv = 5, scoring = 'neg_log_loss')\n",
    "print('logloss of each fold is:' ,loss )\n",
    "print( 'cv logloss is :' ,loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ... C=0.001, penalty=l1, score=-0.6931471805599453, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ... C=0.001, penalty=l1, score=-0.6931471805599453, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ... C=0.001, penalty=l1, score=-0.6931471805599453, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ... C=0.001, penalty=l1, score=-0.6931471805599453, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] ... C=0.001, penalty=l1, score=-0.6931471805599453, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ... C=0.001, penalty=l2, score=-0.6252332751246228, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ... C=0.001, penalty=l2, score=-0.6338686001564596, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] .... C=0.001, penalty=l2, score=-0.625661213962723, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ... C=0.001, penalty=l2, score=-0.6208734221572547, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] ... C=0.001, penalty=l2, score=-0.6326871610823708, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .... C=0.01, penalty=l1, score=-0.6359066018003606, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .... C=0.01, penalty=l1, score=-0.6446073600721886, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .... C=0.01, penalty=l1, score=-0.6332810905191685, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .... C=0.01, penalty=l1, score=-0.6302418522346173, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] .... C=0.01, penalty=l1, score=-0.6358010482921885, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .... C=0.01, penalty=l2, score=-0.5120767708654068, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l2, score=-0.543227572433324, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .... C=0.01, penalty=l2, score=-0.5067380443593051, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .... C=0.01, penalty=l2, score=-0.4873125612462518, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] .... C=0.01, penalty=l2, score=-0.5253888382730233, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .... C=0.1, penalty=l1, score=-0.48922274484805867, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ..... C=0.1, penalty=l1, score=-0.5248837847778534, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .... C=0.1, penalty=l1, score=-0.45824705322340303, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ..... C=0.1, penalty=l1, score=-0.4333601634935699, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] .... C=0.1, penalty=l1, score=-0.48686775745593397, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] .... C=0.1, penalty=l2, score=-0.48350007746296986, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ..... C=0.1, penalty=l2, score=-0.5251494336953129, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ..... C=0.1, penalty=l2, score=-0.4604874856554894, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ..... C=0.1, penalty=l2, score=-0.4292982686216873, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l2, score=-0.485373395496449, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ....... C=1, penalty=l1, score=-0.4881539005209856, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ....... C=1, penalty=l1, score=-0.5291641095987052, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ........ C=1, penalty=l1, score=-0.455620456915051, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ...... C=1, penalty=l1, score=-0.42237599824338207, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ....... C=1, penalty=l1, score=-0.4845262336390694, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ....... C=1, penalty=l2, score=-0.4879785610109463, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ....... C=1, penalty=l2, score=-0.5301159331731353, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ....... C=1, penalty=l2, score=-0.4562291976119336, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ...... C=1, penalty=l2, score=-0.42254600245513574, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ....... C=1, penalty=l2, score=-0.4839288529705514, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ...... C=10, penalty=l1, score=-0.4890368062992649, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ...... C=10, penalty=l1, score=-0.5310440609840115, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ..... C=10, penalty=l1, score=-0.45589055938824524, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ..... C=10, penalty=l1, score=-0.42195585857889284, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ...... C=10, penalty=l1, score=-0.4841106074297348, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ...... C=10, penalty=l2, score=-0.4890294879696647, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ...... C=10, penalty=l2, score=-0.5311363222164094, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ..... C=10, penalty=l2, score=-0.45595951416730635, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ...... C=10, penalty=l2, score=-0.4219795332800078, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ...... C=10, penalty=l2, score=-0.4840864734640913, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ..... C=100, penalty=l1, score=-0.4891379271807341, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ..... C=100, penalty=l1, score=-0.5312325055334342, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] .... C=100, penalty=l1, score=-0.45592833445392944, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ..... C=100, penalty=l1, score=-0.4219231146442935, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ...... C=100, penalty=l1, score=-0.484108990548439, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .... C=100, penalty=l2, score=-0.48914440233874235, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ...... C=100, penalty=l2, score=-0.531246919864776, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .... C=100, penalty=l2, score=-0.45593525922772243, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .... C=100, penalty=l2, score=-0.42192490874569916, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] .... C=100, penalty=l2, score=-0.48410727360689465, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .... C=1000, penalty=l1, score=-0.4891519008528004, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .... C=1000, penalty=l1, score=-0.5312530420573752, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ... C=1000, penalty=l1, score=-0.45593170984937986, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] .... C=1000, penalty=l1, score=-0.4219186094920266, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ... C=1000, penalty=l1, score=-0.48411142398067286, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=-0.489155999776215, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .... C=1000, penalty=l2, score=-0.5312580709718574, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] .... C=1000, penalty=l2, score=-0.4559328627319946, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ... C=1000, penalty=l2, score=-0.42191946802406427, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=-0.484109407281101, total=   0.0s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  70 out of  70 | elapsed:    0.0s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=5)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import  GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty=penaltys,C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters, cv=5 , scoring='neg_log_loss',verbose = 5)\n",
    "grid.fit(X_train,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.47602677786636116\n",
      "{'C': 1, 'penalty': 'l1'}\n"
     ]
    }
   ],
   "source": [
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 以正确率为指标的超参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 14 candidates, totalling 70 fits\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] .... C=0.001, penalty=l1, score=0.6493506493506493, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] .... C=0.001, penalty=l1, score=0.6493506493506493, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] .... C=0.001, penalty=l1, score=0.6493506493506493, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] .... C=0.001, penalty=l1, score=0.6535947712418301, total=   0.0s\n",
      "[CV] C=0.001, penalty=l1 .............................................\n",
      "[CV] .... C=0.001, penalty=l1, score=0.6535947712418301, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] .... C=0.001, penalty=l2, score=0.7337662337662337, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] .... C=0.001, penalty=l2, score=0.7142857142857143, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] .... C=0.001, penalty=l2, score=0.7532467532467533, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] .... C=0.001, penalty=l2, score=0.7843137254901961, total=   0.0s\n",
      "[CV] C=0.001, penalty=l2 .............................................\n",
      "[CV] .... C=0.001, penalty=l2, score=0.7516339869281046, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l1, score=0.6818181818181818, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l1, score=0.6818181818181818, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l1, score=0.7337662337662337, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l1, score=0.7254901960784313, total=   0.0s\n",
      "[CV] C=0.01, penalty=l1 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l1, score=0.7124183006535948, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l2, score=0.7532467532467533, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l2, score=0.7077922077922078, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l2, score=0.7727272727272727, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ...... C=0.01, penalty=l2, score=0.803921568627451, total=   0.0s\n",
      "[CV] C=0.01, penalty=l2 ..............................................\n",
      "[CV] ..... C=0.01, penalty=l2, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l1, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l1, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l1, score=0.7727272727272727, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l1, score=0.7843137254901961, total=   0.0s\n",
      "[CV] C=0.1, penalty=l1 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l1, score=0.7581699346405228, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l2, score=0.7662337662337663, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l2, score=0.7467532467532467, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l2, score=0.7922077922077922, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l2, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=0.1, penalty=l2 ...............................................\n",
      "[CV] ...... C=0.1, penalty=l2, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ........ C=1, penalty=l1, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ........ C=1, penalty=l1, score=0.7467532467532467, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ........ C=1, penalty=l1, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ........ C=1, penalty=l1, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=1, penalty=l1 .................................................\n",
      "[CV] ........ C=1, penalty=l1, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ........ C=1, penalty=l2, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ........ C=1, penalty=l2, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ........ C=1, penalty=l2, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ........ C=1, penalty=l2, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=1, penalty=l2 .................................................\n",
      "[CV] ........ C=1, penalty=l2, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ....... C=10, penalty=l1, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ....... C=10, penalty=l1, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ....... C=10, penalty=l1, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ....... C=10, penalty=l1, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=10, penalty=l1 ................................................\n",
      "[CV] ....... C=10, penalty=l1, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ....... C=10, penalty=l2, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ....... C=10, penalty=l2, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ....... C=10, penalty=l2, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ....... C=10, penalty=l2, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=10, penalty=l2 ................................................\n",
      "[CV] ....... C=10, penalty=l2, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ...... C=100, penalty=l1, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ...... C=100, penalty=l1, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ...... C=100, penalty=l1, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ...... C=100, penalty=l1, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=100, penalty=l1 ...............................................\n",
      "[CV] ...... C=100, penalty=l1, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ...... C=100, penalty=l2, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ...... C=100, penalty=l2, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ...... C=100, penalty=l2, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ...... C=100, penalty=l2, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=100, penalty=l2 ...............................................\n",
      "[CV] ...... C=100, penalty=l2, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ..... C=1000, penalty=l1, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ..... C=1000, penalty=l1, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ..... C=1000, penalty=l1, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ..... C=1000, penalty=l1, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=1000, penalty=l1 ..............................................\n",
      "[CV] ..... C=1000, penalty=l1, score=0.7712418300653595, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=0.7597402597402597, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=0.7402597402597403, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=0.7857142857142857, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=0.7973856209150327, total=   0.0s\n",
      "[CV] C=1000, penalty=l2 ..............................................\n",
      "[CV] ..... C=1000, penalty=l2, score=0.7712418300653595, total=   0.0s\n",
      "0.7747395833333334\n",
      "{'C': 0.1, 'penalty': 'l2'}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    0.0s remaining:    0.0s\n",
      "[Parallel(n_jobs=1)]: Done  70 out of  70 | elapsed:    0.0s finished\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty=penaltys,C=Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters, cv=5, verbose = 5)\n",
    "grid.fit(X_train,y_train)\n",
    "print(grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
